#!/bin/sh

e2e_use_extensions() {
  true
}

e2e_test_install() {
  COPY_CLUSTER_NAME="$(get_sgcluster_name "$CLUSTER_NAME-copy")"
  DISTRIBUTEDLOGS_NAME="$(get_sgdistributedlogs_name distributedlogs)"

  TIMESCALE_VERSION="$(get_latest_version_of_extension timescaledb "${E2E_POSTGRES_VERSION%.*}" || true)"

  if [ -n "$TIMESCALE_VERSION" ]
  then
    create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
      --set distributedLogs.enabled=false \
      --set distributedLogs.create=false \
      --set-string "cluster.postgres.version=$E2E_POSTGRES_VERSION" \
      --set-string 'cluster.postgres.extensions[0].name=timescaledb' \
      --set-string 'cluster.postgres.extensions[0].version='"$TIMESCALE_VERSION" \
      --set-string configurations.postgresconfig.postgresql\\.conf.shared_preload_libraries=timescaledb \
      --set-string 'cluster.managedSql.scripts[0].script=CREATE EXTENSION timescaledb;'
  else
    echo "Extension timescaledb not found [$(uname -m)]"
    create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
      --set distributedLogs.enabled=false \
      --set distributedLogs.create=false \
      --set-string "cluster.postgres.version=$E2E_POSTGRES_VERSION"
  fi

  wait_pods_running "$CLUSTER_NAMESPACE" "1"

  create_or_replace_cluster "$CLUSTER_NAME-for-distributedlogs" "$CLUSTER_NAMESPACE" "1" \
    --set cluster.create=false \
    --set configurations.create=false \
    --set instanceProfiles=null \
    --set distributedLogs.enabled=true \
    --set distributedLogs.create=true \
    --set-string cluster.distributedLogs.sgDistributedLogs="$DISTRIBUTEDLOGS_NAME" \
    --set-string distributedLogs.persistentVolume.size=128Mi

  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
    --set distributedLogs.enabled=true \
    --set distributedLogs.create=false \
    --set-string cluster.distributedLogs.sgDistributedLogs="$CLUSTER_NAMESPACE.$DISTRIBUTEDLOGS_NAME"

  wait_until eval 'kubectl get sts -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" --template="{{ .spec.template.spec.containers | len }}" | grep -qxF 6'
  kubectl delete pod -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME-0"

  deploy_psql_pod "$CLUSTER_NAMESPACE"

  deploy_curl_pod "$CLUSTER_NAMESPACE"

  wait_pods_running "$CLUSTER_NAMESPACE" "4"
  wait_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  wait_cluster "$DISTRIBUTEDLOGS_NAME" "$CLUSTER_NAMESPACE"
}

e2e_test_uninstall() {
  helm_cleanup_chart "$COPY_CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  helm_cleanup_chart "$CLUSTER_NAME" "$CLUSTER_NAMESPACE"

  k8s_async_cleanup_namespace "$CLUSTER_NAMESPACE"
}

e2e_test() {
  if [ -n "$TIMESCALE_VERSION" ]
  then
    run_test "Checking that latest timescaledb version has been installed" check_latest_timescaledb_version_is_installed
  fi

  run_test "Checking that default annotations where created" default_annotations

  run_test "Checking that distributed logs attach to clusters" check_distributed_logs_attach_to_clusters

  run_test "Checking that distributed logs query endpoint works correctly" check_distributed_logs_query_endpoint

  run_test "Checking that retention works correctly" check_retention
}

check_latest_timescaledb_version_is_installed() {
  if wait_until eval '[ "$(kubectl get sgcluster -n "$CLUSTER_NAMESPACE" "$CLUSTER_NAME" -o json \
    | jq "select(.status != null and .status.podStatuses != null) | .status.podStatuses[]
      | select(.installedPostgresExtensions != null) | .installedPostgresExtensions[]
      | select(.name == \"timescaledb\" and .version == \"$TIMESCALE_VERSION\") | length" \
    | jq -s "length")" = 1 ]'
  then
    success "Latest version of timescaledb ($TIMESCALE_VERSION) was installed"
  else
    fail "Latest version of timescaledb ($TIMESCALE_VERSION) was not installed"
  fi
}

default_annotations(){
  local DEFAULT_ANNOTATION
  DEFAULT_ANNOTATION="$(kubectl get sgdistributedlogs.stackgres.io -n "$CLUSTER_NAMESPACE" "$DISTRIBUTEDLOGS_NAME" \
    -o json | jq '.metadata.annotations["stackgres.io/operatorVersion"]')"

  if [ -z "$DEFAULT_ANNOTATION" ] || [ "$DEFAULT_ANNOTATION" = "null" ]
  then
    fail "Default annotations not created"
  else
    success "Defaults annotations created"
  fi
}

check_distributed_logs_attach_to_clusters() {
  if [ -n "$TIMESCALE_VERSION" ]
  then
     create_or_replace_cluster "$COPY_CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
      --set distributedLogs.enabled=true \
      --set distributedLogs.create=false \
      --set instanceProfiles=null \
      --set configurations.create=false \
      --set-string cluster.distributedLogs.sgDistributedLogs="$CLUSTER_NAMESPACE.$DISTRIBUTEDLOGS_NAME" \
      --set-string "cluster.postgres.version=$E2E_POSTGRES_VERSION" \
      --set-string 'cluster.postgres.extensions[0].name=timescaledb' \
      --set-string 'cluster.postgres.extensions[0].version='"$TIMESCALE_VERSION" \
      --set-string configurations.postgresconfig.postgresql\\.conf.shared_preload_libraries=timescaledb \
      --set-string 'cluster.managedSql.scripts[0].script=CREATE EXTENSION timescaledb;'
  else
     create_or_replace_cluster "$COPY_CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
      --set distributedLogs.enabled=true \
      --set distributedLogs.create=false \
      --set instanceProfiles=null \
      --set configurations.create=false \
      --set-string cluster.distributedLogs.sgDistributedLogs="$CLUSTER_NAMESPACE.$DISTRIBUTEDLOGS_NAME" \
      --set-string "cluster.postgres.version=$E2E_POSTGRES_VERSION"
  fi

  wait_pods_running "$CLUSTER_NAMESPACE" "5"
  wait_cluster "$COPY_CLUSTER_NAME" "$CLUSTER_NAMESPACE"
  wait_services_available "$CLUSTER_NAMESPACE" "1" "$CLUSTER_NAME$"
  wait_until -t "$((E2E_TIMEOUT * 4))" has_cluster_joined_distributedlogs
  wait_services_available "$CLUSTER_NAMESPACE" "1" "$COPY_CLUSTER_NAME$"
  wait_until -t "$((E2E_TIMEOUT * 4))" has_cluster_joined_distributedlogs -copy

  local TEXT
  local RESPONSE

  for TEXT in distributed-logs-worked distributed-logs-works distributed-logs-still-works
  do
    RESPONSE="$(run_query -c "$CLUSTER_NAME" -x psql -k postgres \
      -h "$CLUSTER_NAME" -p 5432 -q "SELECT '$TEXT'")"

    if [ "$RESPONSE" = "$TEXT" ]
    then
      success "Log entry '$TEXT' created for $CLUSTER_NAME"
    else
      fail "Cannot create log entry '$TEXT' for $CLUSTER_NAME"
    fi
  done

  for TEXT in distributed-logs-copy-worked distributed-logs-copy-works distributed-logs-copy-still-works
  do
    RESPONSE="$(run_query -c "$COPY_CLUSTER_NAME" -x psql -k postgres \
      -h "$COPY_CLUSTER_NAME" -p 5432 -q "SELECT '$TEXT'")"

    if [ "$RESPONSE" = "$TEXT" ]
    then
      success "Log entry '$TEXT' created for $COPY_CLUSTER_NAME"
    else
      fail "Cannot create log entry '$TEXT' for $COPY_CLUSTER_NAME"
    fi
  done

  wait_services_available "$CLUSTER_NAMESPACE" "1" "$DISTRIBUTEDLOGS_NAME$"

  for TEXT in distributed-logs-worked distributed-logs-works distributed-logs-still-works
  do
    if wait_until eval 'get_logs_count_with_text "$TEXT" | grep -q "^1$"'
    then
      success "Cluster log entry '$TEXT' included in json response for cluster $CLUSTER_NAME"
    else
      fail "Cluster log entry '$TEXT' not included in json response for cluster $CLUSTER_NAME"
    fi
  done

  for TEXT in distributed-logs-copy-worked distributed-logs-copy-works distributed-logs-copy-still-works
  do
    if wait_until eval 'get_logs_count_with_text "$TEXT" -copy | grep -q "^1$"'
    then
      success "Cluster log entry '$TEXT' included in json response for cluster $COPY_CLUSTER_NAME"
    else
      fail "Cluster log entry '$TEXT' not included in json response for cluster $COPY_CLUSTER_NAME"
    fi
  done

  for TEXT in distributed-logs-copy-worked distributed-logs-copy-works distributed-logs-copy-still-works
  do
    if get_logs_count_with_text "$TEXT" | grep -q "^0$"
    then
      success "Cluster log entry '$TEXT' for cluster $COPY_CLUSTER_NAME not included in json response for cluster $CLUSTER_NAME"
    else
      fail "Cluster log entry '$TEXT' for cluster $COPY_CLUSTER_NAME included in json response for cluster $CLUSTER_NAME"
    fi
  done

  for TEXT in distributed-logs-worked distributed-logs-works distributed-logs-still-works
  do
    if get_logs_count_with_text "$TEXT" -copy | grep -q "^0$"
    then
      success "Cluster log entry '$TEXT' for cluster $CLUSTER_NAME not included in json response for cluster $COPY_CLUSTER_NAME"
    else
      fail "Cluster log entry '$TEXT' for cluster $CLUSTER_NAME included in json response for cluster $COPY_CLUSTER_NAME"
    fi
  done
}

has_cluster_joined_distributedlogs() {
  local RESPONSE
  RESPONSE="$(run_query -c "$CLUSTER_NAME$1" -x psql -k postgres \
    -h "$CLUSTER_NAME$1" -p 5432 -q "SELECT 'distributed-logs-joined'")"
  [ "$RESPONSE" = "distributed-logs-joined" ] \
    && get_logs_entries_with_text "distributed-logs-joined" "$1" | jq 'map(select(.role != null)) | length > 0' | grep -q "^true$"
}

check_distributed_logs_query_endpoint() {
  local RESPONSE

  RESPONSE="$(get_logs_count_with_text distributed-logs-works)"
  if echo "$RESPONSE" | grep -q "^1$"
  then
    success "Cluster log entry included in json response"
  else
    fail "Cluster log entry not included in json response"
  fi

  RESPONSE="$(get_logs_entries_with_text distributed-logs-works)"
  if echo "$RESPONSE" | jq 'any(has("logTime"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("logTimeIndex"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(.logType == "pg")' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("podName"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("role"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("errorLevel"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("message"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("userName"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("databaseName"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("processId"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("connectionFrom"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("sessionId"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("sessionLineNum"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("commandTag"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("sessionStartTime"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("virtualTransactionId"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("transactionId"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("sqlStateCode"))' | grep -q true \
    && echo "$RESPONSE" | jq 'any(has("applicationName"))' | grep -q true
  then
    success "Cluster log entry has correct values"
  else
    fail "Cluster log entry has incorrect values: $(echo "$RESPONSE" | jq -c .)"
  fi

  RESPONSE="$(get_logs_count_with_text_after_log_with_text distributed-logs-works distributed-logs-still-works)"
  if echo "$RESPONSE" | grep -q "^1$"
  then
    success "Cluster log entry included in next json response with asc ordering"
  else
    fail "Cluster log entry not included in next json response with asc ordering"
  fi

  RESPONSE="$(get_logs_count_with_text_before_log_with_text distributed-logs-works distributed-logs-worked)"
  if echo "$RESPONSE" | grep -q "^1$"
  then
    success "Cluster log entry included in next json response with desc ordering"
  else
    fail "Cluster log entry not included in next json response with desc ordering"
  fi

  RESPONSE="$(get_logs_count_between_log_with_texts_desc distributed-logs-still-works distributed-logs-works distributed-logs-worked)"
  if echo "$RESPONSE" | grep -q "^2$"
  then
    success "Cluster log entries included in json response with from, to and desc ordering "
  else
    fail "Cluster log entries not included in json response with from, to and desc ordering"
  fi

  RESPONSE="$(get_logs_count_between_log_with_texts_asc distributed-logs-worked distributed-logs-works distributed-logs-still-works)"
  if echo "$RESPONSE" | grep -q "^2$"
  then
    success "Cluster log entries included in json response with from, to and asc ordering"
  else
    fail "Cluster log entries not included in json response with from, to and asc ordering"
  fi

  RESPONSE="$(get_postgres_logs_count_with_text distributed-logs-works)"
  if echo "$RESPONSE" | grep -q "^1$"
  then
    success "Cluster log entry included in json response for postgres"
  else
    fail "Cluster log entry not included in json response for postgres"
  fi

  RESPONSE="$(get_patroni_logs_count)"
  if echo "$RESPONSE" | grep -q "^0$"
  then
    fail "Cluster log entry not included in json response for patroni"
  else
    success "Cluster log entry included in json response for patroni"
  fi

  RESPONSE="$(get_error_and_info_logs_count)"
  if echo "$RESPONSE" | grep -q "^0$"
  then
    fail "Cluster log entry not included in json response for error and info"
  else
    success "Cluster log entry included in json response for error and info"
  fi

  for field in \
    logType \
    podName \
    role \
    errorLevel \
    userName \
    databaseName
  do
    RESPONSE="$(get_logs_count_with_filter_of_log_with_text distributed-logs-works "$field")"
    if echo "$RESPONSE" | grep -q "^1$"
    then
      success "Cluster log entry included in json response with filter on $field"
    else
      fail "Cluster log entry not included in json response with filter on $field"
    fi
  done
}

get_logs_count_with_text() {
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME$2/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | length"
}

get_logs_entries_with_text() {
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME$2/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq "map(select(.message != null and (.message | contains(\"$1\"))))"
}

get_logs_count_with_text_after_log_with_text() {
  local LOG_TIME
  LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?from=$LOG_TIME&sort=asc" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null and (.message | contains(\"$2\")))) | length"
}

get_logs_count_with_text_before_log_with_text() {
  local LOG_TIME
  LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?from=$LOG_TIME&sort=desc" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null and (.message | contains(\"$2\")))) | length"
}

get_logs_count_between_log_with_texts_desc() {
  local START_LOG_TIME
  START_LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1&sort=desc" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local END_LOG_TIME
  END_LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$3&sort=desc" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$3\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?from=$START_LOG_TIME&to=$END_LOG_TIME&sort=desc" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select((.message != null and (.message | contains(\"$1\"))) or (.message != null and (.message | contains(\"$2\"))) or (.message != null and (.message | contains(\"$3\"))))) | length"
}

get_logs_count_between_log_with_texts_asc() {
  local START_LOG_TIME
  START_LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1&sort=asc" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local END_LOG_TIME
  END_LOG_TIME="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$3&sort=asc" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$3\")))) | .[0] | .logTime + \",\" + (.logTimeIndex|tostring)")"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?from=$START_LOG_TIME&to=$END_LOG_TIME&sort=asc" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select((.message != null and (.message | contains(\"$1\"))) or (.message != null and (.message | contains(\"$2\"))) or (.message != null and (.message | contains(\"$3\"))))) | length"
}

get_postgres_logs_count_with_text() {
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1&logType=pg" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | length"
}

get_patroni_logs_count() {
  local FILTER="{\"logType\":\"pa\"}"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?logType=pa" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null)) | length"
}

get_error_and_info_logs_count() {
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?errorLevel=ERROR&errorLevel=LOG&errorLevel=INFO" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null)) | length"
}

get_logs_count_with_filter_of_log_with_text() {
  local FILTER
  FILTER="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")))) | .[0] | .$2 | select(. == null) = \"\"")"
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME/logs?text=$1&$2=$(uriencode "$FILTER")" \
    -n "$CLUSTER_NAMESPACE")"
  echo_raw "$RESULT" \
    | jq -r "map(select(.message != null and (.message | contains(\"$1\")) and (.$2 | select(. == null) = \"\" | tostring) == \"$FILTER\")) | length"
}

check_retention() {
  create_or_replace_cluster "$CLUSTER_NAME" "$CLUSTER_NAMESPACE" "1" \
    --set-string cluster.distributedLogs.retention="1 minute" \
    --set distributedLogs.enabled=true \
    --set distributedLogs.create=false \
    --set-string cluster.distributedLogs.sgDistributedLogs="$CLUSTER_NAMESPACE.$DISTRIBUTEDLOGS_NAME"

  local TABLE
  for TABLE in log_postgres log_patroni
  do
    wait_until run_query -c "$DISTRIBUTEDLOGS_NAME" -x psql -k postgres \
        -h "$DISTRIBUTEDLOGS_NAME" -p 5432 \
        -d "${CLUSTER_NAMESPACE}_${CLUSTER_NAME}" \
        -q "SELECT EXISTS (
        SELECT * FROM _timescaledb_catalog.hypertable
        LEFT JOIN _timescaledb_catalog.dimension ON (hypertable.id = dimension.hypertable_id)
        WHERE table_name = '$TABLE' AND column_name = 'log_time'
        AND INTERVAL '1 minute' = (interval_length / 1000000) * INTERVAL '1 second')"
  done

  for TEXT in distributed-logs-retention-worked distributed-logs-retention-works distributed-logs-retention-still-works
  do
    RESPONSE="$(run_query -c "$CLUSTER_NAME" -x psql -k postgres \
      -h "$CLUSTER_NAME" -p 5432 -q "SELECT '$TEXT'")"

    if [ "$RESPONSE" = "$TEXT" ]
    then
      success "Log entry '$TEXT' created"
    else
      fail "Cannot create log entry '$TEXT'"
    fi
  done

  for TEXT in distributed-logs-retention-worked distributed-logs-retention-works distributed-logs-retention-still-works
  do
    if wait_until eval 'get_logs_count_with_text "$TEXT" | grep -q "^0$"'
    then
      success "Cluster log entry '$TEXT' was removed for cluster $CLUSTER_NAME"
    else
      fail "Cluster log entry '$TEXT' was not removed for cluster $CLUSTER_NAME"
    fi
  done
}

get_logs_timestamp_with_text() {
  local RESULT
  RESULT="$(run_curl \
    -r "stackgres/namespaces/$CLUSTER_NAMESPACE/sgclusters/$CLUSTER_NAME$2/logs?text=$1" \
    -n "$CLUSTER_NAMESPACE")"
  date -d "$(echo_raw "$RESULT" \
    | jq -r ".[] | select(.message != null and (.message | contains(\"$1\"))) | .logTime" \
    | head -n 1 | tr 'T' ' ')" +%s
}
